{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "5760f3c8",
   "metadata": {},
   "source": [
    "## Tabular Income Classifier Kubernetes Test\n",
    "\n",
    "To run this notebook you need the inference data. This can be acquired in two ways:\n",
    "\n",
    "  * Run train.ipynb\n",
    "  * `gsutil cp -R gs://seldon-models/scv2/examples/income/infer-data .`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "2d93f3a1",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ[\"NAMESPACE\"] = \"seldon-mesh\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "c3c33fa0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'172.19.255.1'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "MESH_IP=!kubectl get svc seldon-mesh -n ${NAMESPACE} -o jsonpath='{.status.loadBalancer.ingress[0].ip}'\n",
    "MESH_IP=MESH_IP[0]\n",
    "import os\n",
    "os.environ['MESH_IP'] = MESH_IP\n",
    "MESH_IP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "3af6ea59",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import json\n",
    "import requests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "830f0b2f",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('./infer-data/test.npy', 'rb') as f:\n",
    "    x_ref = np.load(f)\n",
    "    x_h1 = np.load(f)\n",
    "    y_ref = np.load(f)\n",
    "    x_outlier = np.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "346cc929",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'http://172.19.255.1/v2/models/model/infer'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reqJson = json.loads('{\"inputs\":[{\"name\":\"input_1\",\"data\":[],\"datatype\":\"FP32\",\"shape\":[]}]}')\n",
    "url = \"http://\"+MESH_IP+\"/v2/models/model/infer\"\n",
    "url"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "a7339d3d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def infer(resourceName: str, batchSz: int, requestType: str):\n",
    "    if requestType == \"outlier\":\n",
    "        rows = x_outlier[0:0+batchSz]\n",
    "    elif requestType == \"drift\":\n",
    "        rows = x_h1[0:0+batchSz]\n",
    "    else:\n",
    "        rows = x_ref[0:0+batchSz]\n",
    "    reqJson[\"inputs\"][0][\"data\"] = rows.flatten().tolist()\n",
    "    reqJson[\"inputs\"][0][\"shape\"] = [batchSz, rows.shape[1]]\n",
    "    headers = {\"Content-Type\": \"application/json\", \"seldon-model\":resourceName}\n",
    "    response_raw = requests.post(url, json=reqJson, headers=headers)\n",
    "    print(response_raw)\n",
    "    print(response_raw.json())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1f37d87e",
   "metadata": {},
   "source": [
    "### Pipeline with model, drift detector and outlier detector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "c6bbbaa6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model.mlops.seldon.io/income created\r\n"
     ]
    }
   ],
   "source": [
    "!kubectl create -f ../../models/income.yaml -n ${NAMESPACE}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "eac0dbb1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model.mlops.seldon.io/income condition met\r\n"
     ]
    }
   ],
   "source": [
    "!kubectl wait --for condition=ready --timeout=300s model --all -n ${NAMESPACE}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "1f33ac2b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "apiVersion: mlops.seldon.io/v1alpha1\r\n",
      "kind: Pipeline\r\n",
      "metadata:\r\n",
      "  name: income-prod\r\n",
      "spec:\r\n",
      "  steps:\r\n",
      "    - name: income\r\n",
      "  output:\r\n",
      "    steps:\r\n",
      "    - income\r\n",
      "\r\n"
     ]
    }
   ],
   "source": [
    "!cat ../../pipelines/income-v1.yaml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "82194cb3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pipeline.mlops.seldon.io/income-prod configured\r\n"
     ]
    }
   ],
   "source": [
    "!kubectl apply -f ../../pipelines/income-v1.yaml -n ${NAMESPACE}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "20301fd8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pipeline.mlops.seldon.io/income-prod condition met\r\n"
     ]
    }
   ],
   "source": [
    "!kubectl wait --for condition=ready --timeout=300s pipeline --all -n ${NAMESPACE}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "75954454",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\n",
      "<Response [200]>\n",
      "{'model_name': '', 'outputs': [{'data': [0], 'name': 'predict', 'shape': [1], 'datatype': 'INT64'}]}\n"
     ]
    }
   ],
   "source": [
    "batchSz=1\n",
    "print(y_ref[0:batchSz])\n",
    "infer(\"income-prod.pipeline\",batchSz,\"normal\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a5ccce3b",
   "metadata": {},
   "outputs": [],
   "source": [
    "!kubectl create -f ../../models/income-drift.yaml -n ${NAMESPACE}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "b2822b8d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model.mlops.seldon.io/income condition met\n",
      "model.mlops.seldon.io/income-drift condition met\n"
     ]
    }
   ],
   "source": [
    "!kubectl wait --for condition=ready --timeout=300s model --all -n ${NAMESPACE}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "95f0c88a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "apiVersion: mlops.seldon.io/v1alpha1\r\n",
      "kind: Pipeline\r\n",
      "metadata:\r\n",
      "  name: income-prod\r\n",
      "spec:\r\n",
      "  steps:\r\n",
      "    - name: income\r\n",
      "    - name: income-drift\r\n",
      "      batch:\r\n",
      "        size: 20\r\n",
      "  output:\r\n",
      "    steps:\r\n",
      "    - income\r\n",
      "\r\n"
     ]
    }
   ],
   "source": [
    "!cat ../../pipelines/income-v2.yaml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "9d5627f5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Warning: resource pipelines/income-prod is missing the kubectl.kubernetes.io/last-applied-configuration annotation which is required by kubectl apply. kubectl apply should only be used on resources created declaratively by either kubectl create --save-config or kubectl apply. The missing annotation will be patched automatically.\r\n",
      "pipeline.mlops.seldon.io/income-prod configured\r\n"
     ]
    }
   ],
   "source": [
    "!kubectl apply -f ../../pipelines/income-v2.yaml -n ${NAMESPACE}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "7c73b632",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pipeline.mlops.seldon.io/income-prod condition met\r\n"
     ]
    }
   ],
   "source": [
    "!kubectl wait --for condition=ready --timeout=300s pipeline --all -n ${NAMESPACE}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "9c11598d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0 0 1 1 0 1 0 0 1 0 0 0 0 0 1 1 0 0 0 1]\n",
      "<Response [200]>\n",
      "{'model_name': '', 'outputs': [{'data': [0, 0, 1, 1, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1], 'name': 'predict', 'shape': [20], 'datatype': 'INT64'}]}\n"
     ]
    }
   ],
   "source": [
    "batchSz=20\n",
    "print(y_ref[0:batchSz])\n",
    "infer(\"income-prod.pipeline\",batchSz,\"normal\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "3722c9bb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model.mlops.seldon.io/income-preprocess created\n",
      "model.mlops.seldon.io/income-outlier created\n"
     ]
    }
   ],
   "source": [
    "!kubectl create -f ../../models/income-preprocess.yaml -n ${NAMESPACE}\n",
    "!kubectl create -f ../../models/income-outlier.yaml -n ${NAMESPACE}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "2f23e0d9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model.mlops.seldon.io/income condition met\n",
      "model.mlops.seldon.io/income-drift condition met\n",
      "model.mlops.seldon.io/income-outlier condition met\n",
      "model.mlops.seldon.io/income-preprocess condition met\n"
     ]
    }
   ],
   "source": [
    "!kubectl wait --for condition=ready --timeout=300s model --all -n ${NAMESPACE}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "858fb690",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pipeline.mlops.seldon.io/income-prod configured\r\n"
     ]
    }
   ],
   "source": [
    "!kubectl apply -f ../../pipelines/income-v3.yaml -n ${NAMESPACE}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "02184239",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pipeline.mlops.seldon.io/income-prod condition met\r\n"
     ]
    }
   ],
   "source": [
    "!kubectl wait --for condition=ready --timeout=300s pipeline --all -n ${NAMESPACE}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "bc99008f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0 0 1 1 0 1 0 0 1 0 0 0 0 0 1 1 0 0 0 1]\n",
      "<Response [200]>\n",
      "{'model_name': '', 'outputs': [{'data': [0, 0, 1, 1, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 1, 1, 0, 0, 0, 1], 'name': 'predict', 'shape': [20], 'datatype': 'INT64'}, {'data': [1, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, 0, 1, 1], 'name': 'is_outlier', 'shape': [1, 20], 'datatype': 'INT64'}]}\n"
     ]
    }
   ],
   "source": [
    "batchSz=20\n",
    "print(y_ref[0:batchSz])\n",
    "infer(\"income-prod.pipeline\",batchSz,\"outlier\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "c644c816",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0 0 1 1 0 1 0 0 1 0 0 0 0 0 1 1 0 0 0 1]\n",
      "<Response [200]>\n",
      "{'model_name': '', 'outputs': [{'data': [0, 0, 1, 1, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1], 'name': 'predict', 'shape': [20], 'datatype': 'INT64'}, {'data': [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 'name': 'is_outlier', 'shape': [1, 20], 'datatype': 'INT64'}]}\n"
     ]
    }
   ],
   "source": [
    "batchSz=20\n",
    "print(y_ref[0:batchSz])\n",
    "infer(\"income-prod.pipeline\",batchSz,\"normal\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "71019755",
   "metadata": {},
   "source": [
    "### Cleanup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "a2d69d25",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pipeline.mlops.seldon.io \"income-prod\" deleted\r\n"
     ]
    }
   ],
   "source": [
    "!kubectl delete -f ../../pipelines/income-v3.yaml -n ${NAMESPACE}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "ba3ba4ba",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model.mlops.seldon.io \"income\" deleted\n",
      "model.mlops.seldon.io \"income-drift\" deleted\n",
      "model.mlops.seldon.io \"income-preprocess\" deleted\n",
      "model.mlops.seldon.io \"income-outlier\" deleted\n"
     ]
    }
   ],
   "source": [
    "!kubectl delete -f ../../models/income.yaml -n ${NAMESPACE}\n",
    "!kubectl delete -f ../../models/income-drift.yaml -n ${NAMESPACE}\n",
    "!kubectl delete -f ../../models/income-preprocess.yaml -n ${NAMESPACE}\n",
    "!kubectl delete -f ../../models/income-outlier.yaml -n ${NAMESPACE}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "db0ee1f7",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
